{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "9e2667db",
   "metadata": {},
   "source": [
    "# 简介\n",
    "\n",
    "\n",
    "https://blog.csdn.net/qq_49821869/article/details/134493401\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e33b68b2",
   "metadata": {},
   "outputs": [],
   "source": [
    "import evaluate\n",
    "from datasets import DatasetDict,load_from_disk\n",
    "from transformers import AutoTokenizer, AutoModelForMultipleChoice, TrainingArguments, Trainer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "456d5b7b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    test: Dataset({\n",
       "        features: ['id', 'context', 'question', 'choice', 'answer'],\n",
       "        num_rows: 1625\n",
       "    })\n",
       "    train: Dataset({\n",
       "        features: ['id', 'context', 'question', 'choice', 'answer'],\n",
       "        num_rows: 11869\n",
       "    })\n",
       "    validation: Dataset({\n",
       "        features: ['id', 'context', 'question', 'choice', 'answer'],\n",
       "        num_rows: 3816\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c3 = load_from_disk(r'C:\\Users\\caofei\\Desktop\\desktop link\\torch1\\hgface\\4-multiple_choice\\c3')\n",
    "c3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "f8c9c697",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'id': 0,\n",
       " 'context': ['男：你今天晚上有时间吗?我们一起去看电影吧?', '女：你喜欢恐怖片和爱情片，但是我喜欢喜剧片，科幻片一般。所以……'],\n",
       " 'question': '女的最喜欢哪种电影?',\n",
       " 'choice': ['恐怖片', '爱情片', '喜剧片', '科幻片'],\n",
       " 'answer': '喜剧片'}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data1 = c3['train'][0]\n",
    "data1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "f1885c68",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BertTokenizerFast(name_or_path='D:\\models\\chinese-macbert-base', vocab_size=21128, model_max_length=1000000000000000019884624838656, is_fast=True, padding_side='right', truncation_side='right', special_tokens={'unk_token': '[UNK]', 'sep_token': '[SEP]', 'pad_token': '[PAD]', 'cls_token': '[CLS]', 'mask_token': '[MASK]'}, clean_up_tokenization_spaces=False, added_tokens_decoder={\n",
       "\t0: AddedToken(\"[PAD]\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
       "\t100: AddedToken(\"[UNK]\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
       "\t101: AddedToken(\"[CLS]\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
       "\t102: AddedToken(\"[SEP]\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
       "\t103: AddedToken(\"[MASK]\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
       "}\n",
       ")"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_folder = r'D:\\models\\chinese-macbert-base'\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_folder)\n",
    "tokenizer\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "e43b577b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "transformers.models.bert.modeling_bert.BertForMultipleChoice"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "6d3baa01",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'input_ids': [[101, 2769, 4263, 1391, 122, 102, 5741, 3362, 102], [101, 2769, 4263, 1391, 123, 102, 7676, 5933, 102], [101, 2769, 4263, 1391, 124, 102, 3412, 2094, 102]], 'token_type_ids': [[0, 0, 0, 0, 0, 0, 1, 1, 1], [0, 0, 0, 0, 0, 0, 1, 1, 1], [0, 0, 0, 0, 0, 0, 1, 1, 1]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1]]}"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer(text=['我爱吃1','我爱吃2','我爱吃3'],text_pair=['苹果','香蕉','栗子'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "065dd017",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['男：你今天晚上有时间吗?我们一起去看电影吧?',\n",
       " '男：你今天晚上有时间吗?我们一起去看电影吧?',\n",
       " '男：你今天晚上有时间吗?我们一起去看电影吧?',\n",
       " '男：你今天晚上有时间吗?我们一起去看电影吧?']"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "['女的最喜欢哪种电影? 恐怖片', '女的最喜欢哪种电影? 爱情片', '女的最喜欢哪种电影? 喜剧片', '女的最喜欢哪种电影? 科幻片']"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "context = []\n",
    "question_choice = []\n",
    "labels = []\n",
    "\n",
    "for choice in data1['choice']:\n",
    "    context.append(data1['context'][0])\n",
    "    question_choice.append(data1['question']+\" \"+choice)\n",
    "labels = data1['choice'].index(data1['answer'])\n",
    "display(context,question_choice,labels)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "d215f0bb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['女的最喜欢哪种电影? 恐怖片', '女的最喜欢哪种电影? 爱情片', '女的最喜欢哪种电影? 喜剧片', '女的最喜欢哪种电影? 科幻片']"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[ data1['question']+\" \"+_ for _ in data1['choice']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "8b5295d8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['男：你今天晚上有时间吗?我们一起去看电影吧?',\n",
       " '男：你今天晚上有时间吗?我们一起去看电影吧?',\n",
       " '男：你今天晚上有时间吗?我们一起去看电影吧?',\n",
       " '男：你今天晚上有时间吗?我们一起去看电影吧?']"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[ data1['context'][0] for _ in data1['choice']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e5403f32",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'input_ids': tensor([[ 101, 4511, 8038,  872,  791, 1921, 3241,  677, 3300, 3198, 7313, 1408,\n",
       "          136, 2769,  812,  671, 6629, 1343, 4692, 4510, 2512, 1416,  136,  102,\n",
       "         1957, 4638, 3297, 1599, 3614, 1525, 4905, 4510, 2512,  136, 2607, 2587,\n",
       "         4275,  102,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0,    0],\n",
       "        [ 101, 4511, 8038,  872,  791, 1921, 3241,  677, 3300, 3198, 7313, 1408,\n",
       "          136, 2769,  812,  671, 6629, 1343, 4692, 4510, 2512, 1416,  136,  102,\n",
       "         1957, 4638, 3297, 1599, 3614, 1525, 4905, 4510, 2512,  136, 4263, 2658,\n",
       "         4275,  102,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0,    0],\n",
       "        [ 101, 4511, 8038,  872,  791, 1921, 3241,  677, 3300, 3198, 7313, 1408,\n",
       "          136, 2769,  812,  671, 6629, 1343, 4692, 4510, 2512, 1416,  136,  102,\n",
       "         1957, 4638, 3297, 1599, 3614, 1525, 4905, 4510, 2512,  136, 1599, 1196,\n",
       "         4275,  102,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0,    0],\n",
       "        [ 101, 4511, 8038,  872,  791, 1921, 3241,  677, 3300, 3198, 7313, 1408,\n",
       "          136, 2769,  812,  671, 6629, 1343, 4692, 4510, 2512, 1416,  136,  102,\n",
       "         1957, 4638, 3297, 1599, 3614, 1525, 4905, 4510, 2512,  136, 4906, 2404,\n",
       "         4275,  102,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0,    0]]), 'token_type_ids': tensor([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0],\n",
       "        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0],\n",
       "        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0],\n",
       "        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0]]), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0],\n",
       "        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0],\n",
       "        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0],\n",
       "        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0]])}"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 第一种方式\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "15f702b4",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_folder = r'D:\\Models\\chinese-macbert-base'\n",
    "model = AutoModelForMultipleChoice.from_pretrained(model_folder)\n",
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7cff8340",
   "metadata": {},
   "outputs": [],
   "source": [
    "type(model)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "cd7a85b8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 4, 256])"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "tensor([2])"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 转换格式\n",
    "import copy,torch\n",
    "\n",
    "res_txt1 = tokenizer(text=[ data1['context'][0] for _ in data1['choice']],text_pair=[ data1['question']+\" \"+_ for _ in data1['choice']],truncation=True,max_length=256,padding='max_length',return_tensors='pt') # return_tensors='pt'\n",
    "\n",
    "res_txt1\n",
    "\n",
    "\n",
    "res_txt1_copy = copy.deepcopy(res_txt1)\n",
    "res_txt1_copy\n",
    "\n",
    "for k,v in res_txt1_copy.items():\n",
    "    # print(k,v.size())\n",
    "    res_txt1[k] = v.reshape(-1,4,256)\n",
    "\n",
    "res_txt1['labels']=torch.tensor([data1['choice'].index(data1['answer'])])\n",
    "\n",
    "display(res_txt1['input_ids'].shape,res_txt1['labels'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "769df019",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MultipleChoiceModelOutput(loss=tensor(1.3922, grad_fn=<NllLossBackward0>), logits=tensor([[-0.2289, -0.2872, -0.2713, -0.2752]], grad_fn=<ViewBackward0>), hidden_states=None, attentions=None)"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r = model(**res_txt1)\n",
    "r"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "7157e04e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-0.2289, -0.2872, -0.2713, -0.2752]], grad_fn=<ViewBackward0>)"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r.logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "e41eacc5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(1.3922, grad_fn=<NllLossBackward0>)"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch.nn as nn\n",
    "\n",
    "\n",
    "nn.CrossEntropyLoss()(r.logits,res_txt1['labels'])\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "5bf8cd9b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BertForMultipleChoice(\n",
       "  (bert): BertModel(\n",
       "    (embeddings): BertEmbeddings(\n",
       "      (word_embeddings): Embedding(21128, 768, padding_idx=0)\n",
       "      (position_embeddings): Embedding(512, 768)\n",
       "      (token_type_embeddings): Embedding(2, 768)\n",
       "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "      (dropout): Dropout(p=0.1, inplace=False)\n",
       "    )\n",
       "    (encoder): BertEncoder(\n",
       "      (layer): ModuleList(\n",
       "        (0-11): 12 x BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSdpaSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (intermediate_act_fn): GELUActivation()\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (pooler): BertPooler(\n",
       "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "      (activation): Tanh()\n",
       "    )\n",
       "  )\n",
       "  (dropout): Dropout(p=0.1, inplace=False)\n",
       "  (classifier): Linear(in_features=768, out_features=1, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7c8864d7",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a6266e2d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4360ec72",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([4, 256])\n"
     ]
    }
   ],
   "source": [
    "dd = res_txt1['input_ids']\n",
    "dd.size() # 10,4 245\n",
    "dd.view(-1,dd.size(-1))  # 10*4,245\n",
    "print(dd.view(-1,dd.size(-1)).size())\n",
    "\n",
    "\n",
    "\n",
    "bert的结果是  (40,245,768)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dada439a",
   "metadata": {},
   "outputs": [],
   "source": [
    "res = tokenizer(text=context,text_pair=question_choice,truncation=True,max_length=128,padding='max_length') # return_tensors='pt'\n",
    "res\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "21486f13",
   "metadata": {},
   "outputs": [],
   "source": [
    "k2 = {}\n",
    "for k,v in res.items():\n",
    "    # print(v.size())\n",
    "    v2=[]\n",
    "    for i in range(0,len(v),4):\n",
    "        v2 = [v[i:i+4]]\n",
    "    k2[k] = v2\n",
    "    \n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "ef8ae559",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1, 4, 128)"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array(k2['input_ids']).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "1a457921",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([4, 128])"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res['input_ids'].size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "b9613a07",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'input_ids': [[[101,\n",
       "    4511,\n",
       "    8038,\n",
       "    872,\n",
       "    791,\n",
       "    1921,\n",
       "    3241,\n",
       "    677,\n",
       "    3300,\n",
       "    3198,\n",
       "    7313,\n",
       "    1408,\n",
       "    136,\n",
       "    2769,\n",
       "    812,\n",
       "    671,\n",
       "    6629,\n",
       "    1343,\n",
       "    4692,\n",
       "    4510,\n",
       "    2512,\n",
       "    1416,\n",
       "    136,\n",
       "    102,\n",
       "    1957,\n",
       "    4638,\n",
       "    3297,\n",
       "    1599,\n",
       "    3614,\n",
       "    1525,\n",
       "    4905,\n",
       "    4510,\n",
       "    2512,\n",
       "    136,\n",
       "    2607,\n",
       "    2587,\n",
       "    4275,\n",
       "    102,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0],\n",
       "   [101,\n",
       "    4511,\n",
       "    8038,\n",
       "    872,\n",
       "    791,\n",
       "    1921,\n",
       "    3241,\n",
       "    677,\n",
       "    3300,\n",
       "    3198,\n",
       "    7313,\n",
       "    1408,\n",
       "    136,\n",
       "    2769,\n",
       "    812,\n",
       "    671,\n",
       "    6629,\n",
       "    1343,\n",
       "    4692,\n",
       "    4510,\n",
       "    2512,\n",
       "    1416,\n",
       "    136,\n",
       "    102,\n",
       "    1957,\n",
       "    4638,\n",
       "    3297,\n",
       "    1599,\n",
       "    3614,\n",
       "    1525,\n",
       "    4905,\n",
       "    4510,\n",
       "    2512,\n",
       "    136,\n",
       "    4263,\n",
       "    2658,\n",
       "    4275,\n",
       "    102,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0],\n",
       "   [101,\n",
       "    4511,\n",
       "    8038,\n",
       "    872,\n",
       "    791,\n",
       "    1921,\n",
       "    3241,\n",
       "    677,\n",
       "    3300,\n",
       "    3198,\n",
       "    7313,\n",
       "    1408,\n",
       "    136,\n",
       "    2769,\n",
       "    812,\n",
       "    671,\n",
       "    6629,\n",
       "    1343,\n",
       "    4692,\n",
       "    4510,\n",
       "    2512,\n",
       "    1416,\n",
       "    136,\n",
       "    102,\n",
       "    1957,\n",
       "    4638,\n",
       "    3297,\n",
       "    1599,\n",
       "    3614,\n",
       "    1525,\n",
       "    4905,\n",
       "    4510,\n",
       "    2512,\n",
       "    136,\n",
       "    1599,\n",
       "    1196,\n",
       "    4275,\n",
       "    102,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0],\n",
       "   [101,\n",
       "    4511,\n",
       "    8038,\n",
       "    872,\n",
       "    791,\n",
       "    1921,\n",
       "    3241,\n",
       "    677,\n",
       "    3300,\n",
       "    3198,\n",
       "    7313,\n",
       "    1408,\n",
       "    136,\n",
       "    2769,\n",
       "    812,\n",
       "    671,\n",
       "    6629,\n",
       "    1343,\n",
       "    4692,\n",
       "    4510,\n",
       "    2512,\n",
       "    1416,\n",
       "    136,\n",
       "    102,\n",
       "    1957,\n",
       "    4638,\n",
       "    3297,\n",
       "    1599,\n",
       "    3614,\n",
       "    1525,\n",
       "    4905,\n",
       "    4510,\n",
       "    2512,\n",
       "    136,\n",
       "    4906,\n",
       "    2404,\n",
       "    4275,\n",
       "    102,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0]]],\n",
       " 'token_type_ids': [[[0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0],\n",
       "   [0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0],\n",
       "   [0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0],\n",
       "   [0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0]]],\n",
       " 'attention_mask': [[[1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0],\n",
       "   [1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0],\n",
       "   [1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0],\n",
       "   [1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    1,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0,\n",
       "    0]]]}"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "{k: [v[i: i + 4] for i in range(0, len(v), 4)] for k, v in res.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "caaa37e0",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "torch_py10",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
